package fetcher

import (
    "bufio"
    "fmt"
    "golang.org/x/net/html/charset"
    "golang.org/x/text/encoding"
    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
        "io/ioutil"
    "log"
    "net/http"
)

func Fetch(url string) ([]byte, error) {
    reponse, err := http.Get(url)
    if err != nil {
        panic(err)
    }
    
    defer reponse.Body.Close()
    
    if reponse.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("error: status code not OK,wrong status code: %d", reponse.StatusCode)
    }
    
    // utf6Reader := transform.NewReader(reponse.Body, simplifiedchinese.GBK.NewDecoder())
    
    bodyReader := bufio.NewReader(reponse.Body)
    
    analysisEncoding := determineEncoding(bodyReader)
    
    newReader := transform.NewReader(reponse.Body, analysisEncoding.NewDecoder())
    
    return ioutil.ReadAll(newReader)
}

// 根据reader分析编码
func determineEncoding(r *bufio.Reader) encoding.Encoding {
    bytes, err := bufio.NewReader(r).Peek(1024)
    if err != nil {
        log.Printf("Fetcher error: %v", err)
        return unicode.UTF8
    }
    enco, _, _ := charset.DetermineEncoding(bytes, "")
    return enco
}
